diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index c4a015156..cdae54af9 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -112,6 +112,9 @@ class BatchSamplerShard(BatchSampler):
               this argument is set to `False`.
             - the sampler on process 0 to yield `[0, 1]` then `[4, 5]` and the sampler on process 1 to yield `[2, 3]`
               then `[6, 7]` if this argument is set to `True`.
+        even_batches (`bool`, *optional*, defaults to `True`):
+            Whether or not to loop back at the beginning of the sampler when the number of samples is not a round
+            multiple of (original batch size / number of processes).
 
     <Tip warning={true}>
 
@@ -125,6 +128,7 @@ def __init__(
         num_processes: int = 1,
         process_index: int = 0,
         split_batches: bool = False,
+        even_batches: bool = True,
     ):
         if split_batches and batch_sampler.batch_size % num_processes != 0:
             raise ValueError(
@@ -135,8 +139,11 @@ def __init__(
         self.num_processes = num_processes
         self.process_index = process_index
         self.split_batches = split_batches
-        self.batch_size = batch_sampler.batch_size
-        self.drop_last = batch_sampler.drop_last
+        self.even_batches = even_batches
+        self.batch_size = getattr(batch_sampler, "batch_size", None)
+        self.drop_last = getattr(batch_sampler, "drop_last", False)
+        if self.batch_size is None and self.even_batches:
+            raise ValueError("You need to use `even_batches=False` when the batch sampler has no batch size.")
 
     @property
     def total_length(self):
@@ -144,11 +151,21 @@ def total_length(self):
 
     def __len__(self):
         if self.split_batches:
+            # Split batches does not change the length of the batch sampler
             return len(self.batch_sampler)
         if len(self.batch_sampler) % self.num_processes == 0:
+            # If the length is a round multiple of the number of processes, it's easy.
             return len(self.batch_sampler) // self.num_processes
         length = len(self.batch_sampler) // self.num_processes
-        return length if self.drop_last else length + 1
+        if self.drop_last:
+            # Same if we drop the remainder.
+            return length
+        elif self.even_batches:
+            # When we even batches we always get +1
+            return length + 1
+        else:
+            # Otherwise it depends on the process index.
+            return length + 1 if self.process_index < len(self.batch_sampler) % self.num_processes else length
 
     def __iter__(self):
         return self._iter_with_split() if self.split_batches else self._iter_with_no_split()
@@ -165,11 +182,15 @@ def _iter_with_split(self):
 
         # If drop_last is True of the last batch was full, iteration is over, otherwise...
         if not self.drop_last and len(initial_data) > 0 and len(batch) < self.batch_size:
-            # For degenerate cases where the dataset has less than num_process * batch_size samples
-            while len(initial_data) < self.batch_size:
-                initial_data += initial_data
-            batch = batch + initial_data
-            yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)]
+            if not self.even_batches:
+                if len(batch) > batch_length * self.process_index:
+                    yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)]
+            else:
+                # For degenerate cases where the dataset has less than num_process * batch_size samples
+                while len(initial_data) < self.batch_size:
+                    initial_data += initial_data
+                batch = batch + initial_data
+                yield batch[batch_length * self.process_index : batch_length * (self.process_index + 1)]
 
     def _iter_with_no_split(self):
         initial_data = []
@@ -182,35 +203,41 @@ def _iter_with_no_split(self):
             # yielding it.
             if idx % self.num_processes == self.process_index:
                 batch_to_yield = batch
-            if idx % self.num_processes == self.num_processes - 1 and len(batch) == self.batch_size:
+            if idx % self.num_processes == self.num_processes - 1 and (
+                self.batch_size is None or len(batch) == self.batch_size
+            ):
                 yield batch_to_yield
                 batch_to_yield = []
 
         # If drop_last is True, iteration is over, otherwise...
         if not self.drop_last and len(initial_data) > 0:
-            # ... we yield the complete batch we had saved before if it has the proper length
-            if len(batch_to_yield) == self.batch_size:
-                yield batch_to_yield
-
-            # For degenerate cases where the dataset has less than num_process * batch_size samples
-            while len(initial_data) < self.num_processes * self.batch_size:
-                initial_data += initial_data
-
-            # If the last batch seen was of the proper size, it has been yielded by its process so we move to the next
-            if len(batch) == self.batch_size:
-                batch = []
-                idx += 1
-
-            # Make sure we yield a multiple of self.num_processes batches
-            cycle_index = 0
-            while idx % self.num_processes != 0 or len(batch) > 0:
-                end_index = cycle_index + self.batch_size - len(batch)
-                batch += initial_data[cycle_index:end_index]
-                if idx % self.num_processes == self.process_index:
-                    yield batch
-                cycle_index = end_index
-                batch = []
-                idx += 1
+            if not self.even_batches:
+                if len(batch_to_yield) > 0:
+                    yield batch_to_yield
+            else:
+                # ... we yield the complete batch we had saved before if it has the proper length
+                if len(batch_to_yield) == self.batch_size:
+                    yield batch_to_yield
+
+                # For degenerate cases where the dataset has less than num_process * batch_size samples
+                while len(initial_data) < self.num_processes * self.batch_size:
+                    initial_data += initial_data
+
+                # If the last batch seen was of the proper size, it has been yielded by its process so we move to the next
+                if len(batch) == self.batch_size:
+                    batch = []
+                    idx += 1
+
+                # Make sure we yield a multiple of self.num_processes batches
+                cycle_index = 0
+                while idx % self.num_processes != 0 or len(batch) > 0:
+                    end_index = cycle_index + self.batch_size - len(batch)
+                    batch += initial_data[cycle_index:end_index]
+                    if idx % self.num_processes == self.process_index:
+                        yield batch
+                    cycle_index = end_index
+                    batch = []
+                    idx += 1
 
 
 class IterableDatasetShard(IterableDataset):
diff --git a/tests/test_data_loader.py b/tests/test_data_loader.py
index 2d3b6a394..a18dae732 100644
--- a/tests/test_data_loader.py
+++ b/tests/test_data_loader.py
@@ -36,8 +36,11 @@ def __iter__(self):
 
 
 class DataLoaderTester(unittest.TestCase):
-    def check_batch_sampler_shards(self, batch_sampler, expected, split_batches=False):
-        batch_sampler_shards = [BatchSamplerShard(batch_sampler, 2, i, split_batches) for i in range(2)]
+    def check_batch_sampler_shards(self, batch_sampler, expected, split_batches=False, even_batches=True):
+        batch_sampler_shards = [
+            BatchSamplerShard(batch_sampler, 2, i, split_batches=split_batches, even_batches=even_batches)
+            for i in range(2)
+        ]
         batch_sampler_lists = [list(batch_sampler_shard) for batch_sampler_shard in batch_sampler_shards]
         if not split_batches:
             self.assertListEqual([len(shard) for shard in batch_sampler_shards], [len(e) for e in expected])
@@ -164,6 +167,137 @@ def test_batch_sampler_shards_with_splits(self):
         expected = [[], []]
         self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True)
 
+    def test_batch_sampler_shards_with_no_splits_no_even(self):
+        # Check the shards when the dataset is a round multiple of total batch size.
+        batch_sampler = BatchSampler(range(24), batch_size=3, drop_last=False)
+        expected = [
+            [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
+            [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        batch_sampler = BatchSampler(range(24), batch_size=3, drop_last=True)
+        # Expected shouldn't change
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        # Check the shards when the dataset is a round multiple of batch size but not total batch size.
+        batch_sampler = BatchSampler(range(21), batch_size=3, drop_last=False)
+        expected = [
+            [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
+            [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        batch_sampler = BatchSampler(range(21), batch_size=3, drop_last=True)
+        expected = [
+            [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
+            [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
+        # num_processes batch.
+        batch_sampler = BatchSampler(range(22), batch_size=3, drop_last=False)
+        expected = [
+            [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
+            [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        batch_sampler = BatchSampler(range(22), batch_size=3, drop_last=True)
+        expected = [
+            [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
+            [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
+        # num_processes batch.
+        batch_sampler = BatchSampler(range(20), batch_size=3, drop_last=False)
+        expected = [
+            [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
+            [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        batch_sampler = BatchSampler(range(20), batch_size=3, drop_last=True)
+        expected = [
+            [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
+            [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        # Check the shards when the dataset is very small.
+        batch_sampler = BatchSampler(range(2), batch_size=3, drop_last=False)
+        expected = [[[0, 1]], []]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+        batch_sampler = BatchSampler(range(2), batch_size=3, drop_last=True)
+        expected = [[], []]
+        self.check_batch_sampler_shards(batch_sampler, expected, even_batches=False)
+
+    def test_batch_sampler_shards_with_splits_no_even(self):
+        # Check the shards when the dataset is a round multiple of batch size.
+        batch_sampler = BatchSampler(range(24), batch_size=4, drop_last=False)
+        expected = [
+            [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
+            [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True, even_batches=False)
+
+        batch_sampler = BatchSampler(range(24), batch_size=4, drop_last=True)
+        # Expected shouldn't change
+        self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True, even_batches=False)
+
+        # Check the shards when the dataset is not a round multiple of batch size.
+        batch_sampler = BatchSampler(range(22), batch_size=4, drop_last=False)
+        expected = [
+            [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
+            [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True, even_batches=False)
+
+        batch_sampler = BatchSampler(range(22), batch_size=4, drop_last=True)
+        expected = [
+            [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
+            [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True, even_batches=False)
+
+        # Check the shards when the dataset is not a round multiple of batch size or num_processes.
+        batch_sampler = BatchSampler(range(21), batch_size=4, drop_last=False)
+        expected = [
+            [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
+            [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True, even_batches=False)
+
+        batch_sampler = BatchSampler(range(21), batch_size=4, drop_last=True)
+        expected = [
+            [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
+            [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
+        ]
+        self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True, even_batches=False)
+
+        # Check the shards when the dataset is very small.
+        batch_sampler = BatchSampler(range(2), batch_size=4, drop_last=False)
+        expected = [[[0, 1]], []]
+        self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True, even_batches=False)
+
+        batch_sampler = BatchSampler(range(2), batch_size=4, drop_last=True)
+        expected = [[], []]
+        self.check_batch_sampler_shards(batch_sampler, expected, split_batches=True, even_batches=False)
+
+    def test_batch_sampler_with_varying_batch_size(self):
+        batch_sampler = [[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
+        batch_sampler_shards = [BatchSamplerShard(batch_sampler, 2, i, even_batches=False) for i in range(2)]
+
+        self.assertEqual(len(batch_sampler_shards[0]), 3)
+        self.assertEqual(len(batch_sampler_shards[1]), 2)
+
+        self.assertListEqual(list(batch_sampler_shards[0]), [[0, 1, 2], [5, 6, 7, 8], [12, 13]])
+        self.assertListEqual(list(batch_sampler_shards[1]), [[3, 4], [9, 10, 11]])
+
     def check_iterable_dataset_shards(
         self, dataset, seed, batch_size, drop_last=False, num_processes=2, split_batches=False
     ):


Review comments: body: ```suggestion
            Whether or not to loop back at the beginning of the sampler when the number of samples is not a round
            multiple of (original batch size / number of processes).
```
diff_hunk: @@ -112,6 +112,9 @@ class BatchSamplerShard(BatchSampler):
               this argument is set to `False`.
             - the sampler on process 0 to yield `[0, 1]` then `[4, 5]` and the sampler on process 1 to yield `[2, 3]`
               then `[6, 7]` if this argument is set to `True`.
+        even_batches (`bool`, *optional*, defaults to `True`):
+            Whether or not to loop back at the beginning of the sampler when the number of samples is not a round
+            multiple of the batch size/number of processes.
body: `self.batch_size` can't be `None` else this logic would result in errors. 
diff_hunk: @@ -182,35 +201,41 @@ def _iter_with_no_split(self):
             # yielding it.
             if idx % self.num_processes == self.process_index:
                 batch_to_yield = batch
-            if idx % self.num_processes == self.num_processes - 1 and len(batch) == self.batch_size:
+            if idx % self.num_processes == self.num_processes - 1 and (
+                self.batch_size is None or len(batch) == self.batch_size
+            ):
                 yield batch_to_yield
                 batch_to_yield = []
 
         # If drop_last is True, iteration is over, otherwise...
         if not self.drop_last and len(initial_data) > 0:
-            # ... we yield the complete batch we had saved before if it has the proper length
-            if len(batch_to_yield) == self.batch_size:
-                yield batch_to_yield
-
-            # For degenerate cases where the dataset has less than num_process * batch_size samples
-            while len(initial_data) < self.num_processes * self.batch_size:
-                initial_data += initial_data
-
-            # If the last batch seen was of the proper size, it has been yielded by its process so we move to the next
-            if len(batch) == self.batch_size:
-                batch = []
-                idx += 1
-
-            # Make sure we yield a multiple of self.num_processes batches
-            cycle_index = 0
-            while idx % self.num_processes != 0 or len(batch) > 0:
-                end_index = cycle_index + self.batch_size - len(batch)
-                batch += initial_data[cycle_index:end_index]
-                if idx % self.num_processes == self.process_index:
-                    yield batch
-                cycle_index = end_index
-                batch = []
-                idx += 1
+            if not self.even_batches:
+                if len(batch_to_yield) > 0:
+                    yield batch_to_yield
+            else:
+                # ... we yield the complete batch we had saved before if it has the proper length
+                if len(batch_to_yield) == self.batch_size:
+                    yield batch_to_yield
+
+                # For degenerate cases where the dataset has less than num_process * batch_size samples
+                while len(initial_data) < self.num_processes * self.batch_size:
+                    initial_data += initial_data
+
+                # If the last batch seen was of the proper size, it has been yielded by its process so we move to the next
+                if len(batch) == self.batch_size:
+                    batch = []
+                    idx += 1
+
+                # Make sure we yield a multiple of self.num_processes batches
+                cycle_index = 0
+                while idx % self.num_processes != 0 or len(batch) > 0:
+                    end_index = cycle_index + self.batch_size - len(batch)
+                    batch += initial_data[cycle_index:end_index]
+                    if idx % self.num_processes == self.process_index:
+                        yield batch
+                    cycle_index = end_index
+                    batch = []
+                    idx += 1
body: Yes it can't be `None` if `even_batches` is not `False`, will add an early check for that.
diff_hunk: @@ -182,35 +201,41 @@ def _iter_with_no_split(self):
             # yielding it.
             if idx % self.num_processes == self.process_index:
                 batch_to_yield = batch
-            if idx % self.num_processes == self.num_processes - 1 and len(batch) == self.batch_size:
+            if idx % self.num_processes == self.num_processes - 1 and (
+                self.batch_size is None or len(batch) == self.batch_size
+            ):
                 yield batch_to_yield
                 batch_to_yield = []
 
         # If drop_last is True, iteration is over, otherwise...
         if not self.drop_last and len(initial_data) > 0:
-            # ... we yield the complete batch we had saved before if it has the proper length
-            if len(batch_to_yield) == self.batch_size:
-                yield batch_to_yield
-
-            # For degenerate cases where the dataset has less than num_process * batch_size samples
-            while len(initial_data) < self.num_processes * self.batch_size:
-                initial_data += initial_data
-
-            # If the last batch seen was of the proper size, it has been yielded by its process so we move to the next
-            if len(batch) == self.batch_size:
-                batch = []
-                idx += 1
-
-            # Make sure we yield a multiple of self.num_processes batches
-            cycle_index = 0
-            while idx % self.num_processes != 0 or len(batch) > 0:
-                end_index = cycle_index + self.batch_size - len(batch)
-                batch += initial_data[cycle_index:end_index]
-                if idx % self.num_processes == self.process_index:
-                    yield batch
-                cycle_index = end_index
-                batch = []
-                idx += 1
+            if not self.even_batches:
+                if len(batch_to_yield) > 0:
+                    yield batch_to_yield
+            else:
+                # ... we yield the complete batch we had saved before if it has the proper length
+                if len(batch_to_yield) == self.batch_size:
+                    yield batch_to_yield
+
+                # For degenerate cases where the dataset has less than num_process * batch_size samples
+                while len(initial_data) < self.num_processes * self.batch_size:
+                    initial_data += initial_data
+
+                # If the last batch seen was of the proper size, it has been yielded by its process so we move to the next
+                if len(batch) == self.batch_size:
+                    batch = []
+                    idx += 1
+
+                # Make sure we yield a multiple of self.num_processes batches
+                cycle_index = 0
+                while idx % self.num_processes != 0 or len(batch) > 0:
+                    end_index = cycle_index + self.batch_size - len(batch)
+                    batch += initial_data[cycle_index:end_index]
+                    if idx % self.num_processes == self.process_index:
+                        yield batch
+                    cycle_index = end_index
+                    batch = []
+                    idx += 1

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._