diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 514b3a4f4..a79012d5c 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -29,7 +29,7 @@
 from .optimizer import AcceleratedOptimizer
 from .scheduler import AcceleratedScheduler
 from .state import AcceleratorState, DistributedType, is_deepspeed_available
-from .tracking import CometMLTracker, GeneralTracker, TensorBoardTracker, WandBTracker, get_available_trackers
+from .tracking import LOGGER_TYPE_TO_CLASS, GeneralTracker, filter_trackers
 from .utils import (
     DeepSpeedPlugin,
     LoggerType,
@@ -39,9 +39,6 @@
     extract_model_from_parallel,
     gather,
     get_pretty_name,
-    is_comet_ml_available,
-    is_tensorboard_available,
-    is_wandb_available,
     pad_across_processes,
     save,
     wait_for_everyone,
@@ -132,32 +129,13 @@ def __init__(
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
         log_with: Optional[List[Union[str, LoggerType, GeneralTracker]]] = None,
-        logging_dir: Optional[Union[str, os.PathLike]] = "",
+        logging_dir: Optional[Union[str, os.PathLike]] = None,
         dispatch_batches: Optional[bool] = None,
         step_scheduler_with_optimizer: bool = True,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
-        loggers = []
-        if log_with is not None:
-            if not isinstance(log_with, (list, tuple)):
-                log_with = [log_with]
-                logger.debug(f"{log_with}")
-            if "all" in log_with or LoggerType.ALL in log_with:
-                loggers = [o for o in log_with if issubclass(type(o), GeneralTracker)] + get_available_trackers()
-            else:
-                for log_type in log_with:
-                    if log_type not in LoggerType and not issubclass(type(log_type), GeneralTracker):
-                        raise ValueError(
-                            f"Unsupported logging capability: {log_type}. Choose between {LoggerType.list()}"
-                        )
-                    if issubclass(type(log_type), GeneralTracker):
-                        loggers.append(log_type)
-                    else:
-                        log_type = LoggerType(log_type)
-                        if log_type not in loggers:
-                            loggers.append(log_type)
-        self.log_with = loggers
         self.logging_dir = logging_dir
+        self.log_with = filter_trackers(log_with, self.logging_dir)
 
         if mixed_precision is not None:
             mixed_precision = str(mixed_precision)
@@ -644,12 +622,13 @@ def init_trackers(self, project_name: str, config: Optional[dict] = None):
             if issubclass(type(tracker), GeneralTracker):
                 # Custom trackers are already initialized
                 self.trackers.append(tracker)
-            elif str(tracker).lower() == "tensorboard" and is_tensorboard_available():
-                self.trackers.append(TensorBoardTracker(project_name, self.logging_dir))
-            elif str(tracker).lower() == "wandb" and is_wandb_available():
-                self.trackers.append(WandBTracker(project_name))
-            elif str(tracker).lower() == "comet_ml" and is_comet_ml_available():
-                self.trackers.append(CometMLTracker(project_name))
+            else:
+                tracker_init = LOGGER_TYPE_TO_CLASS[str(tracker)]
+                if getattr(tracker_init, "requires_logging_directory"):
+                    # We can skip this check since it was done in `__init__`
+                    self.trackers.append(tracker_init(project_name, self.logging_dir))
+                else:
+                    self.trackers.append(tracker_init(project_name))
         if config is not None:
             for tracker in self.trackers:
                 tracker.store_init_configuration(config)
diff --git a/src/accelerate/tracking.py b/src/accelerate/tracking.py
index 8d496a3d1..de3a19310 100644
--- a/src/accelerate/tracking.py
+++ b/src/accelerate/tracking.py
@@ -17,8 +17,8 @@
 
 import logging
 import os
-from abc import ABCMeta, abstractmethod
-from typing import Optional, Union
+from abc import ABCMeta, abstractmethod, abstractproperty
+from typing import List, Optional, Union
 
 from .utils import LoggerType, is_comet_ml_available, is_tensorboard_available, is_wandb_available
 
@@ -54,6 +54,13 @@ class GeneralTracker(object, metaclass=ABCMeta):
     A base Tracker class to be used for all logging integration implementations.
     """
 
+    @abstractproperty
+    def requires_logging_directory(self):
+        """
+        Whether the logger requires a directory to store their logs. Should either return `True` or `False`.
+        """
+        pass
+
     @abstractmethod
     def store_init_configuration(self, values: dict):
         """
@@ -100,7 +107,9 @@ class TensorBoardTracker(GeneralTracker):
             Location for TensorBoard logs to be stored.
     """
 
-    def __init__(self, run_name: str, logging_dir: Optional[Union[str, os.PathLike]] = ""):
+    requires_logging_directory = True
+
+    def __init__(self, run_name: str, logging_dir: Optional[Union[str, os.PathLike]]):
         self.run_name = run_name
         self.logging_dir = os.path.join(logging_dir, run_name)
         self.writer = tensorboard.SummaryWriter(self.logging_dir)
@@ -157,6 +166,8 @@ class WandBTracker(GeneralTracker):
             The name of the experiment run.
     """
 
+    requires_logging_directory = False
+
     def __init__(self, run_name: str):
         self.run_name = run_name
         self.run = wandb.init(self.run_name)
@@ -209,6 +220,8 @@ class CometMLTracker(GeneralTracker):
             The name of the experiment run.
     """
 
+    requires_logging_directory = False
+
     def __init__(self, run_name: str):
         self.run_name = run_name
         self.writer = Experiment(project_name=run_name)
@@ -250,3 +263,59 @@ def finish(self):
         """
         self.writer.end()
         logger.info("CometML run closed")
+
+
+LOGGER_TYPE_TO_CLASS = {"tensorboard": TensorBoardTracker, "wandb": WandBTracker, "comet_ml": CometMLTracker}
+
+
+def filter_trackers(
+    log_with: List[Union[str, LoggerType, GeneralTracker]], logging_dir: Union[str, os.PathLike] = None
+):
+    """
+    Takes in a list of potential tracker types and checks that:
+        - The tracker wanted is available in that environment
+        - Filters out repeats of tracker types
+        - If `all` is in `log_with`, will return all trackers in the environment
+        - If a tracker requires a `logging_dir`, ensures that `logging_dir` is not `None`
+
+    Args:
+        log_with (list of `str`, [`~utils.LoggerType`] or [`~tracking.GeneralTracker`], *optional*):
+            A list of loggers to be setup for experiment tracking. Should be one or several of:
+
+            - `"all"`
+            - `"tensorboard"`
+            - `"wandb"`
+            - `"comet_ml"`
+            If `"all`" is selected, will pick up all available trackers in the environment and intialize them. Can also
+            accept implementations of `GeneralTracker` for custom trackers, and can be combined with `"all"`.
+        logging_dir (`str`, `os.PathLike`, *optional*):
+            A path to a directory for storing logs of locally-compatible loggers.
+    """
+    loggers = []
+    if log_with is not None:
+        if not isinstance(log_with, (list, tuple)):
+            log_with = [log_with]
+            logger.debug(f"{log_with}")
+        if "all" in log_with or LoggerType.ALL in log_with:
+            loggers = [o for o in log_with if issubclass(type(o), GeneralTracker)] + get_available_trackers()
+        else:
+            for log_type in log_with:
+                if log_type not in LoggerType and not issubclass(type(log_type), GeneralTracker):
+                    raise ValueError(f"Unsupported logging capability: {log_type}. Choose between {LoggerType.list()}")
+                if issubclass(type(log_type), GeneralTracker):
+                    loggers.append(log_type)
+                else:
+                    log_type = LoggerType(log_type)
+                    if log_type not in loggers:
+                        if log_type in get_available_trackers():
+                            tracker_init = LOGGER_TYPE_TO_CLASS[str(log_type)]
+                            if getattr(tracker_init, "requires_logging_directory"):
+                                if logging_dir is None:
+                                    raise ValueError(
+                                        f"Logging with `{str(log_type)}` requires a `logging_dir` to be passed in."
+                                    )
+                            loggers.append(log_type)
+                        else:
+                            logger.info(f"Tried adding logger {log_type}, but package is unavailable in the system.")
+
+    return loggers
diff --git a/tests/test_tracking.py b/tests/test_tracking.py
index b3974afe2..c5b940a46 100644
--- a/tests/test_tracking.py
+++ b/tests/test_tracking.py
@@ -103,6 +103,12 @@ def test_log(self):
         self.assertEqual(iteration, values["iteration"])
         self.assertEqual(my_text, values["my_text"])
 
+    def test_logging_dir(self):
+        with self.assertRaisesRegex(ValueError, "Logging with `tensorboard` requires a `logging_dir`"):
+            _ = Accelerator(log_with="tensorboard")
+        with tempfile.TemporaryDirectory() as dirpath:
+            _ = Accelerator(log_with="tensorboard", logging_dir=dirpath)
+
 
 @mock.patch.dict(os.environ, {"WANDB_MODE": "offline"})
 class WandBTrackingTest(TempDirTestCase, MockingTestCase):
@@ -237,6 +243,8 @@ class MyCustomTracker(GeneralTracker):
         "some_string",
     ]
 
+    requires_logging_directory = False
+
     def __init__(self, dir: str):
         self.f = open(f"{dir}/log.csv", "w+")
         self.writer = csv.DictWriter(self.f, fieldnames=self._col_names)


Review comments: body: Why not use a class attribute directly? We could set a default then (I don't think this actually defaults to False).
diff_hunk: @@ -54,6 +54,14 @@ class GeneralTracker(object, metaclass=ABCMeta):
     A base Tracker class to be used for all logging integration implementations.
     """
 
+    @abstractproperty
+    def requires_logging_directory(self):
+        """
+        Whether the logger requires a directory to store their logs. Should either return `True` or `False`. The
+        default behavior is `False`
+        """
+        pass
body: Very nice!
diff_hunk: @@ -644,12 +653,13 @@ def init_trackers(self, project_name: str, config: Optional[dict] = None):
             if issubclass(type(tracker), GeneralTracker):
                 # Custom trackers are already initialized
                 self.trackers.append(tracker)
-            elif str(tracker).lower() == "tensorboard" and is_tensorboard_available():
-                self.trackers.append(TensorBoardTracker(project_name, self.logging_dir))
-            elif str(tracker).lower() == "wandb" and is_wandb_available():
-                self.trackers.append(WandBTracker(project_name))
-            elif str(tracker).lower() == "comet_ml" and is_comet_ml_available():
-                self.trackers.append(CometMLTracker(project_name))
+            else:
body: This won't right. (This is also evidence of me going back and forth with it). Having it like this IMO forces implementations to think about how they want the behavior to go, and have it explicit in the source code for users interested in the code to see. 

For example, forcing it to be an abstract property/function, W&B could eventually make it a conditional where if it's in `offline_mode` (by checking the env var), then it would be True, otherwise False.

But, I'm not against just making it a normal class attr directly
diff_hunk: @@ -54,6 +54,14 @@ class GeneralTracker(object, metaclass=ABCMeta):
     A base Tracker class to be used for all logging integration implementations.
     """
 
+    @abstractproperty
+    def requires_logging_directory(self):
+        """
+        Whether the logger requires a directory to store their logs. Should either return `True` or `False`. The
+        default behavior is `False`
+        """
+        pass
body: In that case, you should remove the default behavior mention in the docstring, as there is no default :-)
diff_hunk: @@ -54,6 +54,14 @@ class GeneralTracker(object, metaclass=ABCMeta):
     A base Tracker class to be used for all logging integration implementations.
     """
 
+    @abstractproperty
+    def requires_logging_directory(self):
+        """
+        Whether the logger requires a directory to store their logs. Should either return `True` or `False`. The
+        default behavior is `False`
+        """
+        pass
body: Much cleaner!
diff_hunk: @@ -132,32 +129,13 @@ def __init__(
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
         log_with: Optional[List[Union[str, LoggerType, GeneralTracker]]] = None,
-        logging_dir: Optional[Union[str, os.PathLike]] = "",
+        logging_dir: Optional[Union[str, os.PathLike]] = None,
         dispatch_batches: Optional[bool] = None,
         step_scheduler_with_optimizer: bool = True,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
-        loggers = []
-        if log_with is not None:
-            if not isinstance(log_with, (list, tuple)):
-                log_with = [log_with]
-                logger.debug(f"{log_with}")
-            if "all" in log_with or LoggerType.ALL in log_with:
-                loggers = [o for o in log_with if issubclass(type(o), GeneralTracker)] + get_available_trackers()
-            else:
-                for log_type in log_with:
-                    if log_type not in LoggerType and not issubclass(type(log_type), GeneralTracker):
-                        raise ValueError(
-                            f"Unsupported logging capability: {log_type}. Choose between {LoggerType.list()}"
-                        )
-                    if issubclass(type(log_type), GeneralTracker):
-                        loggers.append(log_type)
-                    else:
-                        log_type = LoggerType(log_type)
-                        if log_type not in loggers:
-                            loggers.append(log_type)
-        self.log_with = loggers
         self.logging_dir = logging_dir
+        self.log_with = filter_trackers(log_with, self.logging_dir)

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: @sgugger pinging for a rereview just to make sure the refactor seems sound to you 😄