diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 982293a33..f348bb0a6 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -45,6 +45,7 @@
 )
 from accelerate.utils.constants import DEEPSPEED_MULTINODE_LAUNCHERS
 from accelerate.utils.dataclasses import SageMakerDistributedType
+from accelerate.utils.launch import env_var_path_add
 
 
 if is_rich_available():
@@ -573,7 +574,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
-    current_env["PYTHONPATH"] = sys.executable
+    current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", os.path.abspath("."))
     current_env["MIXED_PRECISION"] = str(mixed_precision)
     current_env["USE_DEEPSPEED"] = "true"
     current_env["DEEPSPEED_ZERO_STAGE"] = str(args.zero_stage)
diff --git a/src/accelerate/utils/launch.py b/src/accelerate/utils/launch.py
index 31d71ad21..8642a441a 100644
--- a/src/accelerate/utils/launch.py
+++ b/src/accelerate/utils/launch.py
@@ -52,6 +52,16 @@ def _filter_args(args):
     return new_args
 
 
+def env_var_path_add(env_var_name, path_to_add):
+    """
+    Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
+    caller to set it in os.environ.
+    """
+    paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0]
+    paths.append(str(path_to_add))
+    return ":".join(paths)
+
+
 class PrepareForLaunch:
     """
     Prepare a function that will launched in a distributed setup.


Review comments: body: Let's just write the three lines of code instead of adding a nested function? It will be shorter and as easy to read.
diff_hunk: @@ -573,7 +573,13 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
-    current_env["PYTHONPATH"] = sys.executable
+    def env_path_add(env_var_name, new_path):
+        """extend path-based env variable with a new path"""
+        paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0]
+        paths.append(str(new_path))
+        return ":".join(paths)
+
+    current_env["PYTHONPATH"] = env_path_add("PYTHONPATH", sys.executable)
body: as it's likely that at some point you might want to extend other path-based env vars like `PATH` or `LD_LIBRARY_PATH` - one more option is to move it to utils. 

but if you prefer inlined and @pacman100 is an agreement I will rewrite.
diff_hunk: @@ -573,7 +573,13 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
-    current_env["PYTHONPATH"] = sys.executable
+    def env_path_add(env_var_name, new_path):
+        """extend path-based env variable with a new path"""
+        paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0]
+        paths.append(str(new_path))
+        return ":".join(paths)
+
+    current_env["PYTHONPATH"] = env_path_add("PYTHONPATH", sys.executable)
body: I'm fine having it as a util too, just don't like the nested function approach where it can't be reused.
diff_hunk: @@ -573,7 +573,13 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
-    current_env["PYTHONPATH"] = sys.executable
+    def env_path_add(env_var_name, new_path):
+        """extend path-based env variable with a new path"""
+        paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0]
+        paths.append(str(new_path))
+        return ":".join(paths)
+
+    current_env["PYTHONPATH"] = env_path_add("PYTHONPATH", sys.executable)
body: pushed the change to move to utils, renamed the helper for better clarity now that it's far away from definition.
diff_hunk: @@ -573,7 +573,13 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
-    current_env["PYTHONPATH"] = sys.executable
+    def env_path_add(env_var_name, new_path):
+        """extend path-based env variable with a new path"""
+        paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0]
+        paths.append(str(new_path))
+        return ":".join(paths)
+
+    current_env["PYTHONPATH"] = env_path_add("PYTHONPATH", sys.executable)
body: ```suggestion
    current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", os.path.abspath('.'))
```
diff_hunk: @@ -573,7 +574,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
-    current_env["PYTHONPATH"] = sys.executable
+    current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", sys.executable)

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Oops, Thanks a lot for pointing this out! Post the above PR, a sample run is shown below:
```
>>> export PYTHONPATH="sample_dir/fancy_library"
...

>>> accelerate launch --config_file ds_mn_hf.yaml accelerate/examples/complete_nlp_example.py --mixed_precision "fp16"
[2022-10-06 12:00:16,739] [INFO] [runner.py:504:main] cmd = pdsh -S -f 1024 -w localhost,sourab_huggingface_co@sourab-vm-image-1 
export PYTHONPATH=sample_dir/fancy_library:/home/sourab_huggingface_co;

...

export MIXED_PRECISION=fp16; export USE_DEEPSPEED=true; export DEEPSPEED_ZERO_STAGE=2; export GRADIENT_ACCUMULATION_STEPS=1; export GRADIENT_CLIPPING=1.0; export DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE=none; export DEEPSPEED_OFFLOAD_PARAM_DEVICE=none; export DEEPSPEED_ZERO3_INIT=false; export DEEPSPEED_ZERO3_SAVE_16BIT_MODEL=none;  cd /home/sourab_huggingface_co; /opt/conda/envs/ml/bin/python3.9 -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMF0sICJzb3VyYWJfaHVnZ2luZ2ZhY2VfY29Ac291cmFiLXZtLWltYWdlLTEiOiBbMF19 --node_rank=%n --master_addr=xx.xxx.x.x --master_port=29500 --no_local_rank accelerate/examples/complete_nlp_example.py --mixed_precision 'fp16'
localhost: [2022-10-06 12:00:22,720] [INFO] [launch.py:136:main] WORLD INFO DICT: {'localhost': [0], 'sourab_huggingface_co@sourab-vm-image-1': [0]}
localhost: [2022-10-06 12:00:22,720] [INFO] [launch.py:142:main] nnodes=2, num_local_procs=1, node_rank=0
localhost: [2022-10-06 12:00:22,720] [INFO] [launch.py:155:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0], 'sourab_huggingface_co@sourab-vm-image-1': [1]})
localhost: [2022-10-06 12:00:22,720] [INFO] [launch.py:156:main] dist_world_size=2
localhost: [2022-10-06 12:00:22,720] [INFO] [launch.py:158:main] Setting CUDA_VISIBLE_DEVICES=0
sourab-vm-image-1: [2022-10-06 12:00:23,404] [INFO] [launch.py:136:main] WORLD INFO DICT: {'localhost': [0], 'sourab_huggingface_co@sourab-vm-image-1': [0]}
sourab-vm-image-1: [2022-10-06 12:00:23,404] [INFO] [launch.py:142:main] nnodes=2, num_local_procs=1, node_rank=1
sourab-vm-image-1: [2022-10-06 12:00:23,404] [INFO] [launch.py:155:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0], 'sourab_huggingface_co@sourab-vm-image-1': [1]})
sourab-vm-image-1: [2022-10-06 12:00:23,404] [INFO] [launch.py:156:main] dist_world_size=2
sourab-vm-image-1: [2022-10-06 12:00:23,404] [INFO] [launch.py:158:main] Setting CUDA_VISIBLE_DEVICES=0
localhost: [2022-10-06 12:00:27,932] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl

...

localhost: [2022-10-06 12:01:10,571] [INFO] [config.py:976:print_user_config]   json = {
localhost:     "train_batch_size": 32, 
localhost:     "train_micro_batch_size_per_gpu": 16, 
localhost:     "gradient_accumulation_steps": 1, 
localhost:     "zero_optimization": {
localhost:         "stage": 2, 
localhost:         "offload_optimizer": {
localhost:             "device": "none"
localhost:         }, 
localhost:         "offload_param": {
localhost:             "device": "none"
localhost:         }, 
localhost:         "stage3_gather_16bit_weights_on_model_save": false
localhost:     }, 
localhost:     "gradient_clipping": 1.0, 
localhost:     "steps_per_print": inf, 
localhost:     "fp16": {
localhost:         "enabled": true, 
localhost:         "auto_cast": true
localhost:     }, 
localhost:     "zero_allow_untested_optimizer": true
localhost: }
localhost: Using /home/sourab_huggingface_co/.cache/torch_extensions/py39_cu113 as PyTorch extensions root...

...

localhost: epoch 0: {'accuracy': 0.7303921568627451, 'f1': 0.8318042813455658}
sourab-vm-image-1: epoch 0: {'accuracy': 0.7303921568627451, 'f1': 0.8318042813455658}
localhost: epoch 1: {'accuracy': 0.8308823529411765, 'f1': 0.880415944540728}
sourab-vm-image-1: epoch 1: {'accuracy': 0.8308823529411765, 'f1': 0.880415944540728}
localhost: epoch 2: {'accuracy': 0.8333333333333334, 'f1': 0.8811188811188811}
sourab-vm-image-1: epoch 2: {'accuracy': 0.8333333333333334, 'f1': 0.8811188811188811}

```
body: @pacman100, please feel free to merge when you feel it's ready. Thanks.