Issue: IterableDataset returns duplicated data using PyTorch DDP

Comment:
@lhoestq I add shuffle after split_dataset_by_node, duplicated data still exist. 
For example, we have a directory named `mock_pretraining_data`, which has three files, `part-00000`, `part-00002`,`part-00002`. 
Text in `part-00000` is like this: 
{"id": 0}
{"id": 1}
{"id": 2}
{"id": 3}
{"id": 4}
{"id": 5}
{"id": 6}
{"id": 7}
{"id": 8}
{"id": 9}

and `part-00001`
{"id": 10}
{"id": 11}
{"id": 12}
{"id": 13}
{"id": 14}
{"id": 15}
{"id": 16}
{"id": 17}
{"id": 18}
{"id": 19}

and `part-00002`
{"id": 20}
{"id": 21}
{"id": 22}
{"id": 23}
{"id": 24}
{"id": 25}
{"id": 26}
{"id": 27}
{"id": 28}
{"id": 29}

And code in `test_dist.py` like this,
```python
import torch
from torch.utils.data import Dataset, DataLoader
from datasets import load_dataset
import os
from transformers import AutoTokenizer, NezhaForPreTraining
from transformers import AdamW, get_linear_schedule_with_warmup
import torch.nn.functional as F
import torch.nn as nn
import torch.distributed as dist
from datasets.distributed import split_dataset_by_node
from torch.nn.parallel import DistributedDataParallel as DDP

os.environ["CUDA_VISIBLE_DEVICES"] = '5,6,7'

dist.init_process_group("nccl")
local_rank = int(os.environ['LOCAL_RANK'])
world_size = torch.distributed.get_world_size()
device = torch.device('cuda', local_rank)
data_dir = './'

def load_trainset(train_path):
    dataset = load_dataset('json', data_dir=os.path.join(data_dir, train_path), split='train', streaming=True)
    return dataset

def collate_fn(examples):
    input_ids = []
    for example in examples:
        input_ids.append(example['id'])
    return torch.LongTensor(input_ids).to(device)


dataset = load_trainset('mock_pretraining_data')
dataset = split_dataset_by_node(dataset, rank=local_rank, world_size=world_size).shuffle(buffer_size=512)
# train_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
batch_size = 3
print('batch_size: {}'.format(batch_size))
train_dataloader = DataLoader(dataset, batch_size=batch_size, collate_fn=collate_fn)

for x in train_dataloader:
    print({'rank': local_rank, 'id': x})
```
run  `python -m torch.distributed.launch --nproc_per_node=3 test_dist.py`
The output is
```
{'rank': 1, 'id': tensor([12, 15, 14], device='cuda:1')}
{'rank': 1, 'id': tensor([16, 10, 18], device='cuda:1')}
{'rank': 1, 'id': tensor([17, 13, 19], device='cuda:1')}
{'rank': 1, 'id': tensor([11], device='cuda:1')}
{'rank': 0, 'id': tensor([0, 2, 9], device='cuda:0')}
{'rank': 0, 'id': tensor([4, 8, 1], device='cuda:0')}
{'rank': 0, 'id': tensor([5, 3, 6], device='cuda:0')}
{'rank': 0, 'id': tensor([7], device='cuda:0')}
{'rank': 2, 'id': tensor([13, 15, 14], device='cuda:2')}
{'rank': 2, 'id': tensor([19, 17, 18], device='cuda:2')}
{'rank': 2, 'id': tensor([12, 16, 11], device='cuda:2')}
{'rank': 2, 'id': tensor([10], device='cuda:2')}
```
`part-00001` is loaded twice, `part-00002` isn't loaded.

If I run `python -m torch.distributed.launch --nproc_per_node=2 test_dist.py`
The output is weirder，many numbers appear twice
```
{'rank': 1, 'id': tensor([26,  8, 13], device='cuda:1')}
{'rank': 1, 'id': tensor([22, 19, 20], device='cuda:1')}
{'rank': 1, 'id': tensor([12, 28, 11], device='cuda:1')}
{'rank': 1, 'id': tensor([24,  2, 14], device='cuda:1')}
{'rank': 1, 'id': tensor([ 6, 27,  3], device='cuda:1')}
{'rank': 0, 'id': tensor([ 8, 25,  1], device='cuda:0')}
{'rank': 0, 'id': tensor([20,  4, 12], device='cuda:0')}
{'rank': 0, 'id': tensor([14, 29,  5], device='cuda:0')}
{'rank': 0, 'id': tensor([ 7, 18, 23], device='cuda:0')}
{'rank': 0, 'id': tensor([19, 17, 11], device='cuda:0')}
```