diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index cdae54af9..8fbc8758f 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -352,16 +352,16 @@ class DataLoaderShard(DataLoader):
         - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes.
     """
 
-    def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwargs):
+    def __init__(self, dataset, device=None, rng_types=None, synchronized_generator=None, **kwargs):
         super().__init__(dataset, **kwargs)
         self.device = device
         self.rng_types = rng_types
-        self.generator = generator
+        self.synchronized_generator = synchronized_generator
         self.gradient_state = GradientState()
 
     def __iter__(self):
         if self.rng_types is not None:
-            synchronize_rng_states(self.rng_types, self.generator)
+            synchronize_rng_states(self.rng_types, self.synchronized_generator)
         self.gradient_state._set_end_of_dataloader(False)
         try:
             length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
@@ -650,12 +650,12 @@ def prepare_data_loader(
     # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
     new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
     sampler_is_batch_sampler = False
-    generator = getattr(dataloader, "generator", None)
+    synchronized_generator = None
     # No change if no multiprocess
     if (num_processes != 1 or state.distributed_type == DistributedType.MEGATRON_LM) and not dispatch_batches:
         if isinstance(new_dataset, IterableDataset):
             if getattr(dataloader.dataset, "generator", None) is not None:
-                generator = dataloader.dataset.generator
+                synchronized_generator = dataloader.dataset.generator
             new_dataset = IterableDatasetShard(
                 new_dataset,
                 batch_size=dataloader.batch_size,
@@ -674,8 +674,7 @@ def prepare_data_loader(
             if hasattr(sampler, "generator"):
                 if sampler.generator is None:
                     sampler.generator = torch.Generator()
-                generator = sampler.generator
-                generator.manual_seed(int(torch.empty((), dtype=torch.int64).random_().item()))
+                synchronized_generator = sampler.generator
 
             batch_sampler = dataloader.sampler if sampler_is_batch_sampler else dataloader.batch_sampler
             new_batch_sampler = BatchSamplerShard(
@@ -692,10 +691,9 @@ def prepare_data_loader(
         "sampler",
         "batch_sampler",
         "drop_last",
-        "generator",
     ]
 
-    if rng_types is not None and generator is None and "generator" in rng_types:
+    if rng_types is not None and synchronized_generator is None and "generator" in rng_types:
         rng_types.remove("generator")
 
     kwargs = {
@@ -710,6 +708,7 @@ def prepare_data_loader(
         kwargs["batch_size"] = dataloader.batch_size // num_processes if split_batches else dataloader.batch_size
 
     if dispatch_batches:
+        kwargs.pop("generator")
         dataloader = DataLoaderDispatcher(
             new_dataset,
             split_batches=split_batches,
@@ -722,9 +721,9 @@ def prepare_data_loader(
             new_dataset,
             device=device if put_on_device and state.distributed_type != DistributedType.TPU else None,
             sampler=new_batch_sampler,
-            batch_size=getattr(dataloader, "batch_size", _PYTORCH_DATALOADER_KWARGS["batch_size"]),
+            batch_size=dataloader.batch_size,
             rng_types=rng_types,
-            generator=generator,
+            synchronized_generator=synchronized_generator,
             **kwargs,
         )
     else:
@@ -733,7 +732,7 @@ def prepare_data_loader(
             device=device if put_on_device and state.distributed_type != DistributedType.TPU else None,
             batch_sampler=new_batch_sampler,
             rng_types=rng_types,
-            generator=generator,
+            synchronized_generator=synchronized_generator,
             **kwargs,
         )
 


Review comments: body: We should have an else branch here in case `self.rng_types` contains something different than `"generator"`, by sending `None` for the generator argument (since we have removed `"generator"` from the rng_types at this stage, if there is none available.
diff_hunk: @@ -361,7 +361,15 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
 
     def __iter__(self):
         if self.rng_types is not None:
-            synchronize_rng_states(self.rng_types, self.generator)
+            batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler
+            sampler = (
+                batch_sampler.batch_sampler.sampler
+                if hasattr(batch_sampler, "batch_sampler")
+                else batch_sampler.sampler
+            )
+            if hasattr(sampler, "generator"):
+                generator = sampler.generator
+                synchronize_rng_states(self.rng_types, generator)
         self.gradient_state._set_end_of_dataloader(False)
body: I wonder why `dataloader.sampler` can be `BatchSampler`?
diff_hunk: @@ -361,7 +361,15 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
 
     def __iter__(self):
         if self.rng_types is not None:
-            synchronize_rng_states(self.rng_types, self.generator)
+            batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler
body: See the example script above in the PR, it has various scenarios. 
diff_hunk: @@ -361,7 +361,15 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
 
     def __iter__(self):
         if self.rng_types is not None:
-            synchronize_rng_states(self.rng_types, self.generator)
+            batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler
body: see https://github.com/huggingface/accelerate/issues/679, it helps reduce the number of queries

diff_hunk: @@ -361,7 +361,15 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
 
     def __iter__(self):
         if self.rng_types is not None:
-            synchronize_rng_states(self.rng_types, self.generator)
+            batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler
body: https://huggingface.co/docs/datasets/v2.4.0/en/use_with_pytorch#use-a-batchsampler
diff_hunk: @@ -361,7 +361,15 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
 
     def __iter__(self):
         if self.rng_types is not None:
-            synchronize_rng_states(self.rng_types, self.generator)
+            batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler
body: Wow! Thanks a lot!
I spent hours finding a way to query my dataset with a list of indices instead of collating data outside the dataset(so that I can write batching logic in the dataset/reduce the number of queries), and failed(thus I was forced to use DataLoader as a sampler(sharded by accelerate), as mentioned in #624 ).
Now I know I can just use batch sampler as sampler without passing batch size to `DataLoader`.
Thanks!!!😁👍
diff_hunk: @@ -361,7 +361,15 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
 
     def __iter__(self):
         if self.rng_types is not None:
-            synchronize_rng_states(self.rng_types, self.generator)
+            batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler
body: We should probably leave this one out and pass it as the generator (as initially suggested in your PR).
diff_hunk: @@ -650,12 +650,12 @@ def prepare_data_loader(
     # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
     new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
     sampler_is_batch_sampler = False
-    generator = getattr(dataloader, "generator", None)
+    synchronized_generator = getattr(dataloader, "generator", None)
body: Here we should pass the main generator of the original dataloader if there was one (see comment above).
diff_hunk: @@ -722,9 +721,9 @@ def prepare_data_loader(
             new_dataset,
             device=device if put_on_device and state.distributed_type != DistributedType.TPU else None,
             sampler=new_batch_sampler,
-            batch_size=getattr(dataloader, "batch_size", _PYTORCH_DATALOADER_KWARGS["batch_size"]),
+            batch_size=dataloader.batch_size,
             rng_types=rng_types,
-            generator=generator,
body: Same there.
diff_hunk: @@ -733,7 +732,7 @@ def prepare_data_loader(
             device=device if put_on_device and state.distributed_type != DistributedType.TPU else None,
             batch_sampler=new_batch_sampler,
             rng_types=rng_types,
-            generator=generator,
body: I remove `"generator"` from `ignore_kwargs`. It should be enough.
diff_hunk: @@ -722,9 +721,9 @@ def prepare_data_loader(
             new_dataset,
             device=device if put_on_device and state.distributed_type != DistributedType.TPU else None,
             sampler=new_batch_sampler,
-            batch_size=getattr(dataloader, "batch_size", _PYTORCH_DATALOADER_KWARGS["batch_size"]),
+            batch_size=dataloader.batch_size,
             rng_types=rng_types,
-            generator=generator,
body: "generator" is removed from `ignore_kwargs`.
diff_hunk: @@ -650,12 +650,12 @@ def prepare_data_loader(
     # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
     new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
     sampler_is_batch_sampler = False
-    generator = getattr(dataloader, "generator", None)
+    synchronized_generator = getattr(dataloader, "generator", None)
body: Yes, but this one should not be synchronized I think. Or do we want to default to the dataloader generator if tehre is no generator in the sampler?
diff_hunk: @@ -650,12 +650,12 @@ def prepare_data_loader(
     # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
     new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
     sampler_is_batch_sampler = False
-    generator = getattr(dataloader, "generator", None)
+    synchronized_generator = getattr(dataloader, "generator", None)
body: Indeed, resolving comment.
diff_hunk: @@ -722,9 +721,9 @@ def prepare_data_loader(
             new_dataset,
             device=device if put_on_device and state.distributed_type != DistributedType.TPU else None,
             sampler=new_batch_sampler,
-            batch_size=getattr(dataloader, "batch_size", _PYTORCH_DATALOADER_KWARGS["batch_size"]),
+            batch_size=dataloader.batch_size,
             rng_types=rng_types,
-            generator=generator,
body: Sorry I didn't get your point. Yes, this line should be `synchronized_generator = None`, as #790 .
diff_hunk: @@ -650,12 +650,12 @@ def prepare_data_loader(
     # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
     new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
     sampler_is_batch_sampler = False
-    generator = getattr(dataloader, "generator", None)
+    synchronized_generator = getattr(dataloader, "generator", None)
body: Oops! Yup, resolved the comment
diff_hunk: @@ -650,12 +650,12 @@ def prepare_data_loader(
     # Iterable dataset doesn't like batch_sampler, but data_loader creates a default one for it
     new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
     sampler_is_batch_sampler = False
-    generator = getattr(dataloader, "generator", None)
+    synchronized_generator = getattr(dataloader, "generator", None)

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: This PR currently will break 
https://github.com/huggingface/accelerate/blob/37b2aa0173fe9b6f46e7df07b92bb55e54b77eb3/src/accelerate/data_loader.py#L653
https://github.com/huggingface/accelerate/blob/37b2aa0173fe9b6f46e7df07b92bb55e54b77eb3/src/accelerate/data_loader.py#L658
https://github.com/huggingface/accelerate/blob/37b2aa0173fe9b6f46e7df07b92bb55e54b77eb3/src/accelerate/data_loader.py#L677
(Of course https://github.com/huggingface/accelerate/blob/37b2aa0173fe9b6f46e7df07b92bb55e54b77eb3/src/accelerate/data_loader.py#L698 should be modified)

---

This code duplication (for finding the sampler) also impairs readability instead of improving readability.
```python
batch_sampler = self.sampler if isinstance(self.sampler, BatchSampler) else self.batch_sampler
sampler = (
    batch_sampler.batch_sampler.sampler
    if hasattr(batch_sampler, "batch_sampler")
    else batch_sampler.sampler
)
if hasattr(sampler, "generator"):
    generator = sampler.generator
    synchronize_rng_states(self.rng_types, generator)
```
I believe what needs to be synchronized is IterableDataset's generator(according to `IterableDatasetShard` implementation) OR sampler's generator. We can store the synchronized generator in a private attribute.

body: @YouJiacheng I am not seeing any breaks in all the lines you mention, which are completely orthogonal to the change suggested. As for the code duplication, let us worry about readability as maintainers :-) We cannot store the generator as a private attribute as it is fetched in two different functions.
body: By "break", I means that these lines will become useless. And for `generator = dataloader.dataset.generator`, it is intended to synchronize the generator of IterableDataset. So after proposed change, IterableDataset's generator won't be synchronized.
body: Ah!, I get what you mean, thanks for clarifying!

We should indeed make the difference between the `sampler_generator` (for batch samplers) or `dataset.generator` (in iterable dataset)  and the Dataloader generator. The latter should be passed along again when creating the final DataLoader (or use the default if it's none) while the formers should be the one synchronized.

The check on `rng_types` should probably be moved to the init of `BatchSamplerShard`/`IterableDatasetShard`.
body: It's hard to explain my proposal using text. So I open a parallel PR to show the code.(It is somewhat "pseudocode"/proof of concept, since I didn't test it).
body: @YouJiacheng Thanks for taking the time to draft a PR to show your points, it's much clearer this way! I think we need to merge the two PRs somehow as they both contain important things the other has not.